package com.chenerzhu.crawler.proxy.pool.job.crawler;

import com.chenerzhu.crawler.proxy.pool.entity.ProxyIp;
import com.chenerzhu.crawler.proxy.pool.job.JsoupUtil;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.util.Date;
import java.util.concurrent.ConcurrentLinkedQueue;

/**
 * @author chenerzhu
 * @create 2018-09-03 20:11
 **/
@Slf4j
public class Data5uCrawlerJob extends AbstractCrawler {
    public Data5uCrawlerJob(ConcurrentLinkedQueue<ProxyIp> proxyIpQueue, String pageUrl) {
        super(proxyIpQueue, pageUrl);
    }

    @Override
    public void parsePage(String url) {
        for (int index = 2; index <= 5; index++) {
            try {
                Document document = JsoupUtil.sendGetRequest(url, timeOut);
                Elements element = document.getElementsByClass("active");
                element.remove(0);
                Elements select = element.select("tr");
                select.remove(0);
                // 遍历每个tr标签
                for (Element tr : select) {
                    // 遍历每个td标签
                    ProxyIp proxyIp = new ProxyIp();
                    String[] s = tr.text().split(" ");
                    proxyIp.setIp(s[0]);
                    proxyIp.setPort(Integer.parseInt(s[1]));
                    proxyIp.setType(s[3]);
                    proxyIp.setCountry(s[s.length - 3] + s[s.length - 2]);
                    proxyIp.setLocation(s[0] + ":" + s[1]);
                    proxyIp.setValidateCount(0);
                    proxyIp.setAvailableCount(0);
                    proxyIp.setUnAvailableCount(0);
                    proxyIp.setAvailableRate(0.0);
                    proxyIp.setAvailable(true);
                    proxyIp.setCreateTime(new Date());
                    proxyIp.setLastValidateTime(new Date());
                    log.info("获取的ip代理================>> {}-->>{}" ,s[0],url);
                    proxyIpQueue.offer(proxyIp);
                }
            }catch (Exception e){
                log.info("请求" + url + "异常,请检查！");
            }
        }
    }
}